#-*-coding:utf-8 -*-
import re
import json
import traceback
from scrapy import log
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy.selector import Selector
from e_commerce_site_crawler_system.items import ECommerceSiteCrawlerSystemItem


class JDSpider(BaseSpider):
    name = "jingdong"
    table_name = "jingdong"
    sku_id_p = re.compile(u"skuid:(.*?),", re.DOTALL)
    price_json_p = re.compile(u"{.*}", re.DOTALL)

    def get_desc_attrib(self, attr, desc_attrib_sel):
        for li_sel in desc_attrib_sel.xpath("//div[@id='product-detail-1']/ul[@class='detail-list']/li"):
            try:
                item = u"".join(li_sel.xpath(".//text()").extract())
                k, v = re.split(u":|：", item, maxsplit=1)
                attr[k.encode("utf-8", "ignore").strip()] = v.strip().encode("utf-8", "ignore")
            except Exception as err:
                continue

    def get_detail_attrib(self, attr, detail_attrib_sel):
        for li_sel in detail_attrib_sel.xpath("//div[@id='product-detail-2']/table[@class='Ptable']/tr"):
            k = u"".join(li_sel.xpath("td[@class='tdTitle']/text()").extract()).strip()
            v = u"".join(li_sel.xpath("td[not(@class='tdTitle')]/text()").extract()).strip()
            if all((k, v)):
                attr[k.encode("utf-8", "ignore").strip()] = v.encode("utf-8", "ignore").strip()

    def get_sku_id(self, html_body):
        sku_id_m = self.sku_id_p.search(html_body)
        if sku_id_m is not None:
            sku_id = sku_id_m.group(1).strip()
            if sku_id.isdigit():
                return sku_id
            else:
                return None
        else:
            return None

    def start_requests(self):
        with open("./jingdong.url") as f:
            index = 0
            for index, line in enumerate(f):
                url = line.strip()
                yield Request(url, meta={"origin_url": url}, dont_filter=True)
            else:
                log.msg("get %s url to crawl of %s" % (index + 1, self.name))

    def parse(self, response):
        meta = response.meta
        meta["sku_id"] = self.get_sku_id(response.body_as_unicode())
        meta["on_shelves"] = 1
        meta["url"] = response.url

        if meta["sku_id"] is None:
            meta["on_shelves"] = 0
            return self.construct_item(meta)

        sel = Selector(response)
        meta["title"] = u"".join(sel.xpath("//div[@id='name']/h1/text()").extract()).strip() or None

        meta["attr"] = {}
        self.get_desc_attrib(meta["attr"], sel)
        self.get_detail_attrib(meta["attr"], sel)
        meta["attr"] = json.dumps(meta["attr"], ensure_ascii=False)

        price_url = "http://d.360buy.com/fittingInfo/get?skuId=%s&callback=Recommend.cbRecoFittings" % meta["sku_id"]
        return Request(price_url, meta=meta, callback=self.parse_price, dont_filter=True)

    def parse_price(self, response):
        meta = response.meta
        price_json_m = self.price_json_p.search(response.body_as_unicode())
        if price_json_m is None:
            return self.construct_item(meta)
        try:
            price_json = price_json_m.group(0)
            price = json.loads(price_json)["master"]["price"].strip()
            meta["price"] = float(price)
            comment_url = "http://club.jd.com/review/%s-0-1-0.html" % meta["sku_id"]
            return Request(comment_url, meta=meta, callback=self.parse_comment, dont_filter=True)
        except Exception as err:
            log.msg(traceback.format_exc(), logLevel=log.WARNING)
            return self.construct_item(meta)

    def get_comment_num(self, sel, xpath):
        num = "".join(sel.xpath(xpath).extract()).strip()
        try:
            return int(num[1:-1])
        except Exception as err:
            return None

    def parse_comment(self, response):
        meta = response.meta
        sel = Selector(response)
        meta["comment_num"] = \
            self.get_comment_num(sel, "//div[@id='comments-list']/div[@class='mt']//li[@scoe='0']/a/em/text()")
        meta["good_comment_num"] = \
            self.get_comment_num(sel, "//div[@id='comments-list']/div[@class='mt']//li[@scoe='3']/a/em/text()")
        meta["medium_comment_num"] = \
            self.get_comment_num(sel, "//div[@id='comments-list']/div[@class='mt']//li[@scoe='2']/a/em/text()")
        meta["bad_comment_num"] = \
            self.get_comment_num(sel, "//div[@id='comments-list']/div[@class='mt']//li[@scoe='1']/a/em/text()")
        return self.construct_item(meta)

    def construct_item(self, info):
        i = ECommerceSiteCrawlerSystemItem()
        i["origin_url"] = info["origin_url"]
        for k in ("url", "on_shelves", "title", "price", "attr",
                  "comment_num", "good_comment_num",
                  "medium_comment_num", "bad_comment_num"):
            i[k] = info.get(k, None)
            if type(i[k]) == unicode:
                i[k] = i[k].encode("utf-8", "ignore")
        return i
